import requests
import json
from lxml import etree


kv = {
"Cookie":""，
"Host":"i.51job.com",
"Referer":"https://search.51job.com/list/000000,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=07&ord_field=0&dibiaoid=0&line=&welfare=",
"User-Agent":""
}
f = open('test.csv','w',encoding='utf-8')
f.write('job_name,workarea_text,company_name,providesalary_text,educational_requirement,hands_on_background,num_recruits,issuedate,companytype_text,companysize_text,jobwelf\n')
for i in range(300):
    r = requests.get('https://search.51job.com/list/000000,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,'+str(i+1)+'.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=',headers=kv)
    tree = etree.HTML(r.text)
    # 所在城市workarea_text、公司名称company_name、薪资providesalary_text、教育要求attribute_text[2]、工作经验要求attribute_text[1]、招聘人数attribute_text[3]、
    # 发布时间issuedate、企业类型companytype_text、企业规模companysize_text、福利待遇jobwelf_list
    jsonData = tree.xpath('/html/body/script/text()')[0][29:]
    datas = json.loads(jsonData)['engine_jds']
    for data in datas:
        job_name=data['job_name'].replace(',','，')
        workarea_text = data['workarea_text'].replace(',','，')
        company_name = data['company_name'].replace(',','，')
        if data['providesalary_text']=='':
            providesalary_text=''
        else:
            providesalary_text=data['providesalary_text']
        attribute_text = data['attribute_text']
        issuedate = data['issuedate']
        companytype_text = data['companytype_text']
        jobwelf = ' '.join(data['jobwelf'].split(','))
        companysize_text = data['companysize_text']
        if len(attribute_text)==3:
            educational_requirement=''
            hands_on_background=''
            num_recruits=attribute_text[-1]
        elif len(attribute_text)==4:
            educational_requirement = attribute_text[2]
            hands_on_background = attribute_text[1]
            if attribute_text[-1]=='':
                num_recruits=''
            else:
                num_recruits = attribute_text[-1]
        else:
            continue
        # print(workarea_text,company_name,providesalary_text,educational_requirement,hands_on_background,num_recruits,issuedate,companytype_text,companysize_text,jobwelf)
        f.write(job_name+','+workarea_text+','+company_name+','+providesalary_text+','+educational_requirement+','+hands_on_background+','+num_recruits+','+issuedate+','+companytype_text+','+companysize_text+','+jobwelf+'\n')
    print('第{}页爬取加载完毕'.format(i+1))
f.close()
